#ifdef BLOCK_DIFFNORM

#include <omp.h>
#include <bitset>
#include <string>
#include <iterator>

#include "../../global.h"
#include "../../qtils/ArrayUtils.h"
#include "../../qtils/StringUtils.h"
#include "DiffNorm.h"
#include "DiffNormUtils.h"
#include "db/DbAnalyser.h"
#include "isc/ItemTranslator.h"

struct CTRecordOrder {
    uint32 ctIdx;
    uint32 numCTs;

    CTRecordOrder(uint32 ctIdx, uint32 numCTs) {
        this->ctIdx = ctIdx;
        this->numCTs = numCTs;
    }

    bool Compare(double cost1, double cost2, ItemSet *is1, ItemSet *is2, uint32 sup1, uint32 sup2) {
        if (cost1 != cost2) return cost1 > cost2;
        if (is1->GetLength() != is2->GetLength()) return is1->GetLength() > is2->GetLength();

        if (sup1 != sup2) return sup1 > sup2;
        return is1->CmpLexicographically(is2) == -1;
    }

    bool operator()(IsRecord* r1, IsRecord* r2) {
        // cardinality - support - lexicographically
        uint32 sup1 = 0, sup2 = 0;
        ItemSet *is1, *is2;
        if (ctIdx == numCTs) {
            is1 = r1->GetItemSetAt(0);
            is2 = r2->GetItemSetAt(0);

            for (uint32 i = 0; i < numCTs; ++i) {
                sup1 += r1->GetItemSetAt(i)->GetSupport();
                sup2 += r2->GetItemSetAt(i)->GetSupport();
            }
        } else {
            is1 = r1->GetItemSetAt(ctIdx);
            is2 = r2->GetItemSetAt(ctIdx);
            sup1 = is1->GetSupport();
            sup2 = is2->GetSupport();
        }
        return Compare(r1->costOfDeletion, r2->costOfDeletion, is1, is2, sup1, sup2);
    }
};

bool CompareEstGain(const DNCandidate *c1, const DNCandidate *c2) {
    return c1->estGain < c2->estGain; // ascending
}

bool CompareEstUsgCount(const DNCandidate *c1, const DNCandidate *c2) {
    return c1->estUsgCount < c2->estUsgCount; // ascending
}

DiffNorm::~DiffNorm() {
    while (!mSingleCTs.empty()) {
        PseudoCT *singleCT = mSingleCTs.back();
        mSingleCTs.pop_back();
        delete singleCT;
    }
}

DiffNorm::DiffNorm(CodeTable** combinedCTs, uint32 numSplits, Config* config) : DiffNormAlgo(combinedCTs, numSplits, config) {
    mCandidate = NULL;
    mMinGain = 1.0; // least 1 bit gain
    mNumIterations = 0;
    mNumRejectedCandidates = 0;
    mNumGeneratedCandidates = 0;
    mAddCandidateToCCT = false;
    mGenerateMoreCandidates = false;
    mNumSingleCTs = mNumDBSplits + 1;
    mNumCombinedCTs = mNumDBSplits;
    mNonSingletonCCTIdx = mNumDBSplits;
    mSingletonCCTIdx = mNumDBSplits + 1;
    
    mMinSup = ExtractMinimumSupport(config->Read<string>("iscname"));
    mReportTranslated = config->Read<bool>("reportTranslated", true);
    mWriteStatsFile = config->Read<bool>("writeStatsFile", true);
    mWriteReportFile = config->Read<bool>("writeReportFile", true);
    mReEstimateAllCandidates = config->Read<bool>("reEstimateAll", false);

    mCandidateGenStrategy = GetCandidateGenerationStrategy(config);
    mRunStrategy = GetDiffNormRunStrategy(config);

    FillHerUp(mCurrentCTSizes, mNumSingleCTs, 0.0);
    FillHerUp(mChangedCTSizes, mNumSingleCTs, 0.0);
    FillHerUp(mCurrentSplitSizes, mNumDBSplits, 0.0);
    FillHerUp(mChangedSplitSizes, mNumDBSplits, 0.0);
}

bool DiffNorm::AccceptedBefore(const string& dnid) {
    return mAccepted.find(dnid) != mAccepted.end();
}

void DiffNorm::AcceptCandidate() {
    mAccepted.insert(mCandidate->GetDNID());
    mGenerateMoreCandidates = true;
    
    double prevSize = mCurTotalSize;
    mCurTotalSize = CalcTotalEncodedSizeFromIndividualSizes();
    assert(mCurTotalSize < prevSize);
    
    // we need the changed sizes for prunning
    ShallowCopy(mChangedCTSizes, mCurrentCTSizes);
    ShallowCopy(mChangedSplitSizes, mCurrentSplitSizes);
}

void DiffNorm::AddCandidateToAllCTs() {
    if (mCandidate->leftCTIdx == mCandidate->rightCTIdx && mCandidate->leftCTIdx == mNonSingletonCCTIdx) {
        // CT_Omega + CT_Omega
        IsRecord *recordForCCT = new IsRecord(mNumCombinedCTs);
        uint32 i;
        for (i = 0; i < mNumCombinedCTs; ++i) {
            IsRecord *recordForSpecificCT = new IsRecord(mNumCombinedCTs);
            ItemSet *isClone = mCandidate->mergedIs->Clone();

            recordForSpecificCT->SetItemSetAt(isClone, i);
            recordForCCT->SetItemSetAt(isClone, i);

            mCombinedCTs[i]->Add(isClone);
            mCombinedCTs[i]->CoverDB(mCombinedCTs[i]->GetCurStats());
            mSingleCTs[i]->Add(recordForSpecificCT);

            mChangedSplitSizes[i] = mCombinedCTs[i]->CalcEncodedDbSize();
            mChangedCTSizes[i] = CalcEncodedSizeOfCTFromSingleCT(i);
        }
        // For Common Code Table
        mSingleCTs[i]->Add(recordForCCT);
        mChangedCTSizes[i] = CalcEncodedSizeOfCTFromSingleCT(i);
    } else {
        uint32 ctToAdd;
        if (mCandidate->leftCTIdx == mCandidate->rightCTIdx && mCandidate->leftCTIdx != mNonSingletonCCTIdx)
            ctToAdd = mCandidate->leftCTIdx;    // CT_i + CT_i
        else
            ctToAdd = mCandidate->leftCTIdx != mNonSingletonCCTIdx ? mCandidate->leftCTIdx : mCandidate->rightCTIdx;    // CT_i + CT_Omega
        
        IsRecord *recordForSpecificCT = new IsRecord(mNumCombinedCTs);
        ItemSet *isClone = mCandidate->mergedIs->Clone();
        recordForSpecificCT->SetItemSetAt(isClone, ctToAdd);
        
        mCombinedCTs[ctToAdd]->Add(isClone);
        mCombinedCTs[ctToAdd]->CoverDB(mCombinedCTs[ctToAdd]->GetCurStats());
        mSingleCTs[ctToAdd]->Add(recordForSpecificCT);
        
        mChangedSplitSizes[ctToAdd] = mCombinedCTs[ctToAdd]->CalcEncodedDbSize();
        mChangedCTSizes[ctToAdd] = CalcEncodedSizeOfCTFromSingleCT(ctToAdd);
    } 
}

bool DiffNorm::AlreadyGenerated(const string& id) {
    return mGenerated.find(id) != mGenerated.end();
}

double DiffNorm::CalcCompressionTime() {
    return mCompressionEndTime - mCompressionStartTime;
}

double DiffNorm::CalcCostOfDeletion(uint32 ctIdx, uint32 recordIdx) {
    double costOfDeletion = 0.0;
    IsRecord *record = mSingleCTs[ctIdx]->records[recordIdx];
    if (ctIdx == mNonSingletonCCTIdx) {
        for (uint32 i = 0; i < mNumCombinedCTs; ++i) {
            mCombinedCTs[i]->Del(record->GetItemSetAt(i), false, false);
            mCombinedCTs[i]->CoverDB(mCombinedCTs[i]->GetCurStats());
            costOfDeletion += mCombinedCTs[i]->CalcEncodedDbSize() - mCurrentSplitSizes[i];
            mCombinedCTs[i]->RollbackLastDel();
        }
    } else {
        mCombinedCTs[ctIdx]->Del(record->GetItemSetAt(ctIdx), false, false);
        mCombinedCTs[ctIdx]->CoverDB(mCombinedCTs[ctIdx]->GetCurStats());
        costOfDeletion += mCombinedCTs[ctIdx]->CalcEncodedDbSize() - mCurrentSplitSizes[ctIdx];
        mCombinedCTs[ctIdx]->RollbackLastDel();
    }
    
    recordsItr pos = mSingleCTs[ctIdx]->records.begin() + recordIdx;
    mSingleCTs[ctIdx]->Del(pos, false);
    costOfDeletion += CalcEncodedSizeOfCTFromSingleCT(ctIdx) - mCurrentCTSizes[ctIdx];
    mSingleCTs[ctIdx]->RollbackLastDel();
    return costOfDeletion;
}

double DiffNorm::CalcEncodedSizeOfCTFromSingleCT(uint32 singleCTIdx) {
    PseudoCT *singleCT = mSingleCTs[singleCTIdx];
    double size = UniversalCodeLength(singleCT->NumRecords());
    uint32 isIdx = singleCTIdx;
    if (singleCTIdx == mNonSingletonCCTIdx) isIdx = 0;
    
    for (recordsItr itr = singleCT->records.begin(); itr != singleCT->records.end(); ++itr) {
        ItemSet *is = (*itr)->GetItemSetAt(isIdx);
        uint32 cardinality = is->GetLength();
        size += UniversalCodeLength(cardinality) + CalcEncodedSizeOfItemSetByST(is);
    }
    return size;
}

double DiffNorm::CalcEncodedSizeOfItemSetByST(ItemSet* is) {
    mDB->CalcStdLength(is);
    return is->GetStandardLength();
}

double DiffNorm::CalcExpectation(ItemSet* is) {
    double expectation = 1.0;
    uint16 *alphs = is->GetValues();
    uint32 *supps = mDB->GetAlphabetNumRows();
    uint32 numRows = mDB->GetNumRows();
    for (uint32 i = 0; i < is->GetLength(); ++i) {
        expectation = expectation * supps[alphs[i]] / numRows;
    }
    return expectation;
}

double DiffNorm::CalcGainFromIndividualSizes() {
    double gain = 0.0;
    for (uint32 i = 0; i < mNumDBSplits + 1; ++i) {
        gain += mCurrentCTSizes[i] - mChangedCTSizes[i];
        if (i != mNonSingletonCCTIdx)
            gain += mCurrentSplitSizes[i] - mChangedSplitSizes[i];
    }
    return gain;
}

void DiffNorm::CalcImportanceOfItemSetsInAllCTs() {
    double cost;
    for (uint32 i = 0; i < mNumDBSplits + 1; ++i) {
        for (uint16 j = 0; j < mSingleCTs[i]->NumRecords(); ++j) {
            cost = CalcCostOfDeletion(i, j);
            mSingleCTs[i]->records[j]->SetCostOfDeletion(cost);
        }
    }
}

double DiffNorm::CalcTotalEncodedSize() {
    double totalSize = 0.0;
    for (uint32 i = 0; i < mNumDBSplits + 1; ++i) {
        double ctSize = CalcEncodedSizeOfCTFromSingleCT(i);
        totalSize += ctSize;
        mCurrentCTSizes[i] = ctSize;

        if (i != mNonSingletonCCTIdx) {
            // calculate l(d_i | ct_i + ct_omega) from combined code tables
            double splitSize = mCombinedCTs[i]->CalcEncodedDbSize();
            totalSize += splitSize;
            mCurrentSplitSizes[i] = splitSize;
        }
    }
    return totalSize;
}

double DiffNorm::CalcTotalEncodedSizeFromIndividualSizes() {
    double size = 0.0;
    for (uint32 i = 0; i < mNumDBSplits + 1; ++i) {
        size += mCurrentCTSizes[i];
        if (i != mNonSingletonCCTIdx) size += mCurrentSplitSizes[i];
    }
    return size;
}

void DiffNorm::CheckGainInEachCTs() {
    mCandidateAccepted = false;
    mAddCandidateToCCT = false;
    mCommitCTIndices.clear();

    vector<double> gains;
    double dbGain = 0.0, ctGain = 0.0, totalGain = 0.0, maxGain = 0.0;
    double totalDbGain = 0.0, totalGainAtSpecificCTs = 0.0;

    if (mCandidate->leftCTIdx == mCandidate->rightCTIdx && mCandidate->leftCTIdx == mNonSingletonCCTIdx) {    
        for (uint32 i = 0; i < mNumSingleCTs; ++i) {
            ctGain = mCurrentCTSizes[i] - mChangedCTSizes[i];

            if (i == mNonSingletonCCTIdx) {
                dbGain = totalDbGain;
            } else {
                dbGain = mCurrentSplitSizes[i] - mChangedSplitSizes[i];
                totalDbGain += dbGain;
            }

            totalGain = dbGain + ctGain;
            gains.push_back(totalGain);
            if (totalGain < mMinGain) continue;

            mCandidateAccepted = true;
            if (i == mNonSingletonCCTIdx) {
                if (mNumDBSplits == 1 || totalGain >= totalGainAtSpecificCTs) {
                    mCommitCTIndices.clear();
                    mAddCandidateToCCT = true;
                }
            } else {
                totalGainAtSpecificCTs += totalGain;
                mCommitCTIndices.push_back(i);
            }
        }

        if (mCandidateAccepted) maxGain = max(totalGainAtSpecificCTs, totalGain);
        else                    maxGain = *max_element(gains.begin(), gains.end());
    } else {
        uint32 ctAdded;
        if (mCandidate->leftCTIdx == mCandidate->rightCTIdx && mCandidate->leftCTIdx != mNonSingletonCCTIdx)
            ctAdded = mCandidate->leftCTIdx;    // CT_i + CT_i
        else
            ctAdded = mCandidate->leftCTIdx != mNonSingletonCCTIdx ? mCandidate->leftCTIdx : mCandidate->rightCTIdx;    // CT_i + CT_Omega
        
        maxGain = mCurrentSplitSizes[ctAdded] - mChangedSplitSizes[ctAdded] + mCurrentCTSizes[ctAdded] - mChangedCTSizes[ctAdded];
        if (maxGain >= mMinGain) {
            mCandidateAccepted = true;
            mCommitCTIndices.push_back(ctAdded);
        }
    }
    mCandidate->SetActualGain(maxGain);
}

void DiffNorm::CollectItemSetRecordsInCCT(vector<IsRecord*> &records) {
    vector<IsRecord*> singletonRecords, nonSingletonRecords;
    nonSingletonRecords = mSingleCTs.at(mNonSingletonCCTIdx)->records;
    singletonRecords = mSingleCTs.at(mSingletonCCTIdx)->records;

    records.reserve(singletonRecords.size() + nonSingletonRecords.size());
    records.insert(records.end(), singletonRecords.begin(), singletonRecords.end());
    records.insert(records.end(), nonSingletonRecords.begin(), nonSingletonRecords.end());
}

void DiffNorm::CommitAcceptedChanges() {
    if (mAddCandidateToCCT) {
        for (uint32 i=0; i<mNumCombinedCTs; ++i) 
            CommitChangesToCombinedCT(i);
        CommitChangesToSingleCT(mNonSingletonCCTIdx);
    } else {
        for (uint32 i = 0; i < mCommitCTIndices.size(); ++i) {
            CommitChangesToCombinedCT(mCommitCTIndices[i]);
            CommitChangesToSingleCT(mCommitCTIndices[i]);
        }
    }
}

void DiffNorm::CommitChangesToCombinedCT(uint32 ctIdx) {
    mCombinedCTs[ctIdx]->CommitAdd();
    mCurrentSplitSizes[ctIdx] = mChangedSplitSizes[ctIdx];
}

void DiffNorm::CommitChangesToSingleCT(uint32 ctIdx) {
    mCurrentCTSizes[ctIdx] = mChangedCTSizes[ctIdx];
}

void DiffNorm::CreateMergedId(const string &leftId, const string &rightId, string &mergedId) {
    // mergedId is empty if length of the merged itemset is greater than the 
    // maximum length of transaction in the database
    if (leftId == rightId)
        return;
    
    mergedId.reserve(50);
    mergedId = (leftId + "-") + rightId;
    stringstream ss(mergedId);
    
    vector<uint32> tokens;
    tokens.reserve(20);
    
    string item;
    item.reserve(5);
    
    while (getline(ss, item, '-')) {
        uint32 token = atoi(item.c_str());
        if (find(tokens.begin(), tokens.end(), token) == tokens.end())
            tokens.push_back(token);
    }
    sort(tokens.begin(), tokens.end());

    mergedId = "";
    if (tokens.size() <= mDB->GetMaxSetLength()) {
        for (uint32 i = 0; i < tokens.size(); ++i) {
#if _MSC_VER <= 1600
            mergedId += to_string((unsigned long long) tokens[i]);
#else
			mergedId += to_string(tokens[i]);
#endif
            if (i != tokens.size() - 1)
                mergedId += "-";
        }
    }
}

double DiffNorm::CalcDbGain(DNCandidate *candidate, vector<uint32> &xyUsgList, vector<uint32> &xUsgList, vector<uint32> &yUsgList, vector<uint32> &xpUsgList, vector<uint32> &ypUsgList, uint32 combinedCtIdx) {
    double dbGain, e = 0.5;
    uint32 m, mc, mp, mx, my, mxp, myp, mxy, l, lp;
    
    GetUsageList(candidate->leftRecord, xUsgList, combinedCtIdx);
    GetUsageList(candidate->rightRecord, yUsgList, combinedCtIdx);
    
    set_intersection(xUsgList.begin(), xUsgList.end(), yUsgList.begin(), yUsgList.end(), back_inserter(xyUsgList));
    set_difference(xUsgList.begin(), xUsgList.end(), xyUsgList.begin(), xyUsgList.end(), back_inserter(xpUsgList));
    set_difference(yUsgList.begin(), yUsgList.end(), xyUsgList.begin(), xyUsgList.end(), back_inserter(ypUsgList));

    mc = mCombinedCTs[combinedCtIdx]->GetSumUsageCounts();
    mx = xUsgList.size();
    my = yUsgList.size();
    mxy = xyUsgList.size();
    mxp = xpUsgList.size();
    myp = ypUsgList.size();

    m = mc;
    mp = mc - mx - my + mxp + myp + mxy;

    l = mCombinedCTs[combinedCtIdx]->GetNumItemSets();
    lp = l + 1;

    dbGain = LogGamma(m + e * l) - LogGamma(e * l) - LogGamma(mx + e) -
            LogGamma(my + e) - LogGamma(mp + e * lp) + LogGamma(e * lp) +
            LogGamma(mxp + e) + LogGamma(myp + e) + LogGamma(mxy + e) -
            LogGamma(e);
    
    xyUsgList.clear();
    xUsgList.clear();
    yUsgList.clear();
    xpUsgList.clear();
    ypUsgList.clear();
    
    return dbGain;
}

void DiffNorm::EstimateGain(DNCandidate *candidate) {
    // calc gain only for concerned code tables and use the max gain
    double gain, maxGain, dbGain, cctGain;
    double totalDbGainInOtherCTs = 0.0, totalPositiveGainsInOtherCTs = 0.0;

    vector<double> gains;
    vector<uint32> xyUsgList, xUsgList, yUsgList, xpUsgList, ypUsgList;
    double ctGain = UniversalCodeLength(candidate->GetCardinality()) + CalcEncodedSizeOfItemSetByST(candidate->mergedIs);

    if (candidate->leftCTIdx == candidate->rightCTIdx && candidate->leftCTIdx == mNonSingletonCCTIdx) {
        for (uint32 i=0; i<mNumCombinedCTs; ++i) {
            dbGain = CalcDbGain(candidate, xyUsgList, xUsgList, yUsgList, xpUsgList, ypUsgList, i);
            gain = dbGain + ctGain;
            gains.push_back(gain);
            totalDbGainInOtherCTs += dbGain;
            
            if (gain > 0.0) totalPositiveGainsInOtherCTs += gain;
        }
        cctGain = totalDbGainInOtherCTs + ctGain;
        if (totalPositiveGainsInOtherCTs > 0.0) maxGain = max(totalPositiveGainsInOtherCTs, cctGain);
        else                                    maxGain = *max_element(gains.begin(), gains.end());
    } else {
        uint32 ctToCheck;
        if (candidate->leftCTIdx == candidate->rightCTIdx && candidate->leftCTIdx != mNonSingletonCCTIdx)
            ctToCheck = candidate->leftCTIdx;    // CT_i + CT_i
        else
            ctToCheck = candidate->leftCTIdx != mNonSingletonCCTIdx ? candidate->leftCTIdx : candidate->rightCTIdx;    // CT_i + CT_Omega
        
        dbGain = CalcDbGain(candidate, xyUsgList, xUsgList, yUsgList, xpUsgList, ypUsgList, ctToCheck);
        maxGain = dbGain + ctGain;
    }
    candidate->SetEstimatedGain(maxGain);
}

uint32 DiffNorm::EstimateMaxUsage(IsRecord* record1, IsRecord* record2, uint32 ctIdx1, uint32 ctIdx2) {
    uint32 estMaxUsg;
    uint32 minLen = mMinSup - 1;
    if (ctIdx1 == mNonSingletonCCTIdx && ctIdx2 == mNonSingletonCCTIdx) {
        vector<uint32> usg1, usg2;
        GatherUsagesForCommonCT(record1, usg1);
        GatherUsagesForCommonCT(record2, usg2);
        estMaxUsg = ArrayUtils::IntersectionLength(usg1.data(), usg1.size(), usg2.data(), usg2.size(), minLen);
    } else if (ctIdx1 == mNonSingletonCCTIdx) {
        vector<uint32> usg1;
        GatherUsagesForCommonCT(record1, usg1);
        ItemSet *is2 = record2->GetItemSetAt(ctIdx2);

        estMaxUsg = ArrayUtils::IntersectionLength(usg1.data(), usg1.size(), GetUsages(is2, ctIdx2), GetUsageCount(is2, ctIdx2), minLen);
    } else if (ctIdx2 == mNonSingletonCCTIdx) {
        vector<uint32> usg2;
        ItemSet *is1 = record1->GetItemSetAt(ctIdx1);
        GatherUsagesForCommonCT(record2, usg2);

        estMaxUsg = ArrayUtils::IntersectionLength(GetUsages(is1, ctIdx1), GetUsageCount(is1, ctIdx1), usg2.data(), usg2.size(), minLen);
    } else {
        ItemSet *is1 = record1->GetItemSetAt(ctIdx1);
        ItemSet *is2 = record2->GetItemSetAt(ctIdx2);

        estMaxUsg = ArrayUtils::IntersectionLength(GetUsages(is1, ctIdx1), GetUsageCount(is1, ctIdx1), GetUsages(is2, ctIdx2), GetUsageCount(is2, ctIdx2), minLen);
    }
    return estMaxUsg;
}

uint32 DiffNorm::ExtractMinimumSupport(string iscName) {
    string minSup = iscName.substr(iscName.find_last_of("-") + 1, iscName.find_last_of("d") - iscName.find_last_of("-") - 1);
    return stoi(minSup);
}

void DiffNorm::GetRecords(uint32 ctIdx, vector<IsRecord*> &records) {
    if (ctIdx == mNonSingletonCCTIdx) {
        CollectItemSetRecordsInCCT(records);
    } else {
        records.reserve(mSingleCTs[ctIdx]->NumRecords());
        records.insert(records.begin(), mSingleCTs[ctIdx]->records.begin(), mSingleCTs[ctIdx]->records.end());
    }
}

uint32 DiffNorm::GetItemSetIdx(uint32 idx) {
    return idx == mNonSingletonCCTIdx ? 0 : idx;
}

void DiffNorm::GatherUsagesForCommonCT(IsRecord* record, vector<uint32>& usages) {
    for (uint32 i = 0; i < mNumCombinedCTs; ++i) {
        ItemSet *is = record->GetItemSetAt(i);
        uint32 *usgs = GetUsages(is, i);
        uint32 usgCount = GetUsageCount(is, i);
        for (uint32 j = 0; j < usgCount; ++j)
            usages.push_back(usgs[j]);
    }
}

void DiffNorm::GenerateCandidates() {
    if (mCandidateGenStrategy == GAIN)  GenerateCandidatesBasedOnGain();
    else                                GenerateCandidatesBasedOnUsage();
}

void DiffNorm::GenerateCandidateBasedOnGain(IsRecord* leftRecord, IsRecord* rightRecord, ItemSet *leftIs, ItemSet *rightIs, uint32 leftCtIdx, uint32 rightCtIdx) {
    string mergedId;
    CreateMergedId(leftIs->GetDNID(), rightIs->GetDNID(), mergedId);
    if (mergedId.empty() || RejectedBefore(mergedId) || AccceptedBefore(mergedId) || AlreadyGenerated(mergedId))
        return;

    uint32 estMaxUsgCount = EstimateMaxUsage(leftRecord, rightRecord, leftCtIdx, rightCtIdx);
    ++mNumGeneratedCandidates;
    if (estMaxUsgCount > mMinSup) {
        DNCandidate *candidate = new DNCandidate(leftCtIdx, rightCtIdx, leftRecord, rightRecord, mNumCombinedCTs, mergedId, 0.0, estMaxUsgCount, 0.0);
        EstimateGain(candidate);
        mGenerated.insert(mergedId);
        mCandidates.push_back(candidate);
    } else {
        mRejected.insert(mergedId);
    }
}

void DiffNorm::GenerateCandidatesBasedOnGain() {
    vector<IsRecord*> recordsOmega;
    GetRecords(mNonSingletonCCTIdx, recordsOmega);
    uint32 omegaIsIdx = GetItemSetIdx(mNonSingletonCCTIdx);
    if (mNumIterations == 0) {    
        // CT_Omega x CT_Omega
        for (uint32 i=0; i<recordsOmega.size(); ++i) {
            ItemSet *leftIs = recordsOmega[i]->GetItemSetAt(omegaIsIdx);
            for (uint32 j=i+1; j<recordsOmega.size(); ++j) {
                ItemSet *rightIs  = recordsOmega[j]->GetItemSetAt(omegaIsIdx);
                GenerateCandidateBasedOnGain(recordsOmega[i], recordsOmega[j], leftIs, rightIs, mNonSingletonCCTIdx, mNonSingletonCCTIdx);
            }
        }
    } else if (mAddCandidateToCCT) {
        IsRecord *leftRecord = recordsOmega.back();
        ItemSet *addedIs = leftRecord->GetItemSetAt(omegaIsIdx);
        for (uint32 k=0; k<mNumDBSplits; ++k) {
            vector<IsRecord*> rj;
            GetRecords(k, rj);
            
            // CT_i x CT_Omega
            for (uint32 j=0; j<rj.size(); ++j) {
                ItemSet *rightIs = rj[j]->GetItemSetAt(k);
                GenerateCandidateBasedOnGain(leftRecord, rj[j], addedIs, rightIs, mNonSingletonCCTIdx, k);
            }
        }
        
        // CT_Omega x CT_Omega
        for (uint32 j=0; j<recordsOmega.size() - 1; ++j) {
            ItemSet *rightIs = recordsOmega[j]->GetItemSetAt(omegaIsIdx);
            GenerateCandidateBasedOnGain(leftRecord, recordsOmega[j], addedIs, rightIs, mNonSingletonCCTIdx, mNonSingletonCCTIdx);
        }
    } else {
        for (uint32 k=0; k<mCommitCTIndices.size(); ++k) {
            uint32 changedCTIdx = mCommitCTIndices[k];
            vector<IsRecord*> ri;
            GetRecords(changedCTIdx, ri);
            
            IsRecord *leftRecord = ri.back();
            ItemSet *leftIs = leftRecord->GetItemSetAt(changedCTIdx);
            
            // CT_i x CT_i
            for (uint32 i=0; i<ri.size()-1; ++i) {
                ItemSet *rightIs = ri[i]->GetItemSetAt(changedCTIdx);
                GenerateCandidateBasedOnGain(leftRecord, ri[i], leftIs, rightIs, changedCTIdx, changedCTIdx);
            }
            
            // CT_i x CT_Omega
            for (uint32 i=0; i<recordsOmega.size(); ++i) {
                ItemSet *rightIs = recordsOmega[i]->GetItemSetAt(omegaIsIdx);
                GenerateCandidateBasedOnGain(leftRecord, recordsOmega[i], leftIs, rightIs, changedCTIdx, mNonSingletonCCTIdx);
            }
        }
    }
    
    ++mNumIterations;
    sort(mCandidates.begin(), mCandidates.end(), CompareEstGain);
    if (mCandidates.empty()) mGenerateMoreCandidates = false;
}

void DiffNorm::GenerateCandidateBasedOnUsage(IsRecord* leftRecord, IsRecord* rightRecord, ItemSet* leftIs, ItemSet* rightIs, uint32 leftCtIdx, uint32 rightCtIdx) {
    string mergedId;
    CreateMergedId(leftIs->GetDNID(), rightIs->GetDNID(), mergedId);
    if (mergedId.empty() || RejectedBefore(mergedId) || AccceptedBefore(mergedId) || AlreadyGenerated(mergedId))
        return;

    uint32 estMaxUsgCount = EstimateMaxUsage(leftRecord, rightRecord, leftCtIdx, rightCtIdx);
    ++mNumGeneratedCandidates;
    if (estMaxUsgCount > mMinSup) {
        DNCandidate *candidate = new DNCandidate(leftCtIdx, rightCtIdx, leftRecord, rightRecord, mNumCombinedCTs, mergedId, 0.0, estMaxUsgCount, 0.0);
        mGenerated.insert(mergedId);
        mCandidates.push_back(candidate);
    } else {
        mRejected.insert(mergedId);
    }
}

void DiffNorm::GenerateCandidatesBasedOnUsage() {
    vector<IsRecord*> recordsOmega;
    GetRecords(mNonSingletonCCTIdx, recordsOmega);
    uint32 omegaIsIdx = GetItemSetIdx(mNonSingletonCCTIdx);
    
    for (uint32 k=0; k<mNumDBSplits; ++k) {
        vector<IsRecord*> ri;
        GetRecords(k, ri);
        
        for (uint32 i = 0; i<ri.size(); ++i) {
            ItemSet *leftIs = ri[i]->GetItemSetAt(k);
            // CT_i x CT_i
            for (uint32 j=i+1; j<ri.size(); ++j) {
                ItemSet *rightIs = ri[j]->GetItemSetAt(k);
                GenerateCandidateBasedOnUsage(ri[i], ri[j], leftIs, rightIs, k, k);
            }
            
            // CT_i x CT_Omega
            for (uint32 j=0; j<recordsOmega.size(); ++j) {
                ItemSet *rightIs = recordsOmega[j]->GetItemSetAt(omegaIsIdx);
                GenerateCandidateBasedOnUsage(ri[i], recordsOmega[j], leftIs, rightIs, k, mNonSingletonCCTIdx);
            }
        }
    }
    
    // CT_Omega x CT_Omega
    for (uint32 i=0; i<recordsOmega.size(); ++i) {
        ItemSet *leftIs = recordsOmega[i]->GetItemSetAt(omegaIsIdx);
        for (uint32 j=i+1; j<recordsOmega.size(); ++j) {
            ItemSet *rightIs  = recordsOmega[j]->GetItemSetAt(omegaIsIdx);
            GenerateCandidateBasedOnUsage(recordsOmega[i], recordsOmega[j], leftIs, rightIs, mNonSingletonCCTIdx, mNonSingletonCCTIdx);
        }
    }
    
    ++mNumIterations;
    sort(mCandidates.begin(), mCandidates.end(), CompareEstUsgCount);
    if (!mCandidates.empty()) mGenerateMoreCandidates = false;
}

CandidateGenerationStrategy DiffNorm::GetCandidateGenerationStrategy(Config* config) {
    CandidateGenerationStrategy genStrategy;
    string strategy = config->Read<string>("genStrategy", "gain");

    if (strategy == "gain") genStrategy = GAIN;
    else if (strategy == "usg") genStrategy = USAGE;
    else THROW("Unknown candidate estimation estrategy!!!");

    return genStrategy;
}

DiffNormRunStrategy DiffNorm::GetDiffNormRunStrategy(Config* config) {
    DiffNormRunStrategy runStrategy;
    string strategy = config->Read<string>("runStrategy", "rcpa");

    if (strategy == "rcpa") runStrategy = RCPA; // regenerate candidates post acceptance
    else if (strategy == "rcta") runStrategy = RCTA; // regenerate candidates after testing all
    else THROW("Unknown run strategy!!!");

    return runStrategy;
}

uint32 DiffNorm::GetNumAcceptedCandidates() {
    return mAccepted.size();
}

uint32 DiffNorm::GetNumRejectedCandidates() {
    return mRejected.size();
}

uint32 DiffNorm::GetPrevUsageCount(ItemSet* is, uint32 ctIdx) {
    assert(ctIdx < mNumCombinedCTs);
    // if singleton, get the prev usage count from the respective code table
    return is->GetLength() == 1 ? mCombinedCTs[ctIdx]->GetOldAlphabetCount(is->GetUniqueID()) : is->GetPrevUsageCount();
}

uint32 DiffNorm::GetPrevUsageCount(IsRecord* record, uint32 singleCTIdx) {
    uint32 prevUsageCount = 0;
    if (singleCTIdx == mNonSingletonCCTIdx) {
        for (uint32 i = 0; i < mNumCombinedCTs; ++i)
            prevUsageCount += GetPrevUsageCount(record->GetItemSetAt(i), i);
    } else {
        prevUsageCount = GetPrevUsageCount(record->GetItemSetAt(singleCTIdx), singleCTIdx);
    }
    return prevUsageCount;
}

uint32 DiffNorm::GetUsageCount(IsRecord* record, uint32 singleCTIdx) {
    uint32 usageCount = 0;
    if (singleCTIdx == mNonSingletonCCTIdx) {
        for (uint32 i = 0; i < mNumCombinedCTs; ++i)
            usageCount += GetUsageCount(record->GetItemSetAt(i), i);
    } else {
        usageCount += GetUsageCount(record->GetItemSetAt(singleCTIdx), singleCTIdx);
    }
    return usageCount;
}

uint32 DiffNorm::GetUsageCount(ItemSet* is, uint32 ctIdx) {
    assert(ctIdx < mNumCombinedCTs);
    // if singleton, get the usage count from the respective code table
    return is->GetLength() == 1 ? mCombinedCTs[ctIdx]->GetAlphabetCount(is->GetUniqueID()) : is->GetUsageCount();
}

uint32* DiffNorm::GetUsages(ItemSet* is, uint32 ctIdx) {
    assert(ctIdx < mNumCombinedCTs);
    // if singleton, get the usages from the respective code table
    return is->GetLength() == 1 ? mCombinedCTs[ctIdx]->GetAlphabetUsage(is->GetUniqueID()) : is->GetUsages();
}

void DiffNorm::GetUsageList(IsRecord *record, uint32 ctIdx, vector<uint32> &usgList) {
    if (ctIdx == mNonSingletonCCTIdx) {
        for (uint32 i = 0; i < mNumCombinedCTs; ++i) {
            ItemSet *is = record->GetItemSetAt(i);
            uint32 *usgs = GetUsages(is, i);
            for (uint32 j = 0; j < GetUsageCount(is, i); ++j)
                usgList.push_back(usgs[j]);
        }
    } else {
        ItemSet *is = record->GetItemSetAt(ctIdx);
        uint32 *usgs = GetUsages(is, ctIdx);
        for (uint32 i = 0; i < GetUsageCount(is, ctIdx); ++i)
            usgList.push_back(usgs[i]);
    }
}

void DiffNorm::GetUsageList(IsRecord* record, vector<uint32>& usgList, uint32 ctToAdd) {
    assert(ctToAdd < mNumCombinedCTs);

    ItemSet *is = record->GetItemSetAt(ctToAdd);
    uint32 *usgs = GetUsages(is, ctToAdd);
    for (uint32 i = 0; i < GetUsageCount(is, ctToAdd); ++i)
        usgList.push_back(usgs[i]);
}

void DiffNorm::InitializeSingleCTs() {
    mSingleCTs.reserve(mNumDBSplits + 2);
    for (uint32 i = 0; i < mNumDBSplits + 1; ++i) {
        mSingleCTs.push_back(new PseudoCT());
    }
    UpdateSingletonsCCT();
    
    mTotalSizeInStart = CalcTotalEncodedSize();
    mCurTotalSize = mTotalSizeInStart;
    
    ShallowCopy(mChangedCTSizes, mCurrentCTSizes);
    ShallowCopy(mChangedSplitSizes, mCurrentSplitSizes);
}

uint8 DiffNorm::IsZero(uint32 val) {
    return val == 0 ? 1 : 0;
}

void DiffNorm::LoadNextCandidate() {
    mCandidate = mCandidates.back();
    mCandidates.pop_back();
}

void DiffNorm::NoteDownCompressionStartTime() {
    mCompressionStartTime = omp_get_wtime();
}

void DiffNorm::NoteDownCompressionEndTime() {
    mCompressionEndTime = omp_get_wtime();
    printf(" * Time:    \t\tCompressing the database took %f seconds.\t\t\n", CalcCompressionTime());
}

void DiffNorm::PostPrune() {
    if (mAddCandidateToCCT) {
        for (uint32 i=0; i<mNumCombinedCTs; ++i)
            mCommitCTIndices.push_back(i);
    }
        
    bool prunned;
    for (uint32 i = 0; i < mCommitCTIndices.size(); ++i) {
        while (true) {
            prunned = PruneSpecificCT(mCommitCTIndices[i]);
            if (!prunned)   break;
        }
    }
    
    while (true) {
        prunned = PruneCommonCT();
        if (!prunned)   break;
    }
    
    mCurTotalSize = CalcTotalEncodedSizeFromIndividualSizes();
}

bool DiffNorm::PruneCommonCT() {
    bool pruneAgain = false;
    recordsItr itr = mSingleCTs[mNonSingletonCCTIdx]->records.begin();
    while (itr != mSingleCTs[mNonSingletonCCTIdx]->records.end()) {
        IsRecord *record = *itr;
        
        uint32 prevUsageCount = GetPrevUsageCount(record, mNonSingletonCCTIdx);
        uint32 curUsageCount = GetUsageCount(record, mNonSingletonCCTIdx);

        if (curUsageCount == 0) {
            RemoveCandidateFromPool(record->dnid);

            for (uint32 k = 0; k < mNumCombinedCTs; ++k) {
                mCombinedCTs[k]->Del(record->GetItemSetAt(k), true, false);
                mCurrentSplitSizes[k] = mCombinedCTs[k]->CalcEncodedDbSize();
            }

            itr = mSingleCTs[mNonSingletonCCTIdx]->Del(itr, true);
            mCurrentCTSizes[mNonSingletonCCTIdx] = CalcEncodedSizeOfCTFromSingleCT(mNonSingletonCCTIdx);
        } else if (curUsageCount < prevUsageCount) {

            for (uint32 k = 0; k < mNumCombinedCTs; ++k) {
                mCombinedCTs[k]->Del(record->GetItemSetAt(k), false, false);
                mCombinedCTs[k]->CoverDB(mCombinedCTs[k]->GetCurStats());
                mChangedSplitSizes[k] = mCombinedCTs[k]->CalcEncodedDbSize();
            }

            itr = mSingleCTs[mNonSingletonCCTIdx]->Del(itr, false);
            mChangedCTSizes[mNonSingletonCCTIdx] = CalcEncodedSizeOfCTFromSingleCT(mNonSingletonCCTIdx);
            if (CalcGainFromIndividualSizes() >= 0) {
                pruneAgain = true;
                RemoveCandidateFromPool(record->dnid);

                for (uint32 k = 0; k < mNumCombinedCTs; ++k) {
                    mCombinedCTs[k]->CommitLastDel(true);
                    mCurrentSplitSizes[k] = mChangedSplitSizes[k];
                }

                mSingleCTs[mNonSingletonCCTIdx]->CommitLastDel();
                mCurrentCTSizes[mNonSingletonCCTIdx] = mChangedCTSizes[mNonSingletonCCTIdx];
            } else {
                for (uint32 k = 0; k < mNumCombinedCTs; ++k) {
                    mCombinedCTs[k]->RollbackLastDel();
                    mChangedSplitSizes[k] = mCurrentSplitSizes[k];
                }
                mSingleCTs[mNonSingletonCCTIdx]->RollbackLastDel();
                mChangedCTSizes[mNonSingletonCCTIdx] = mCurrentCTSizes[mNonSingletonCCTIdx];
                ++itr;
            }
        } else {
            ++itr;
        }
    }
    return pruneAgain;
}

bool DiffNorm::PruneSpecificCT(uint32 ctIdx) {
    bool pruneAgain = false;
    recordsItr itr = mSingleCTs[ctIdx]->records.begin();
    while (itr != mSingleCTs[ctIdx]->records.end()) {
        IsRecord *record = *itr;
        uint32 prevUsageCount = GetPrevUsageCount(record, ctIdx);
        uint32 curUsageCount = GetUsageCount(record, ctIdx);
        
        if (curUsageCount == 0) {
            RemoveCandidateFromPool(record->dnid);

            mCombinedCTs[ctIdx]->Del(record->GetItemSetAt(ctIdx), true, false);
            mCurrentSplitSizes[ctIdx] = mCombinedCTs[ctIdx]->CalcEncodedDbSize();

            itr = mSingleCTs[ctIdx]->Del(itr, true);
            mCurrentCTSizes[ctIdx] = CalcEncodedSizeOfCTFromSingleCT(ctIdx);
        } else if (curUsageCount < prevUsageCount) {
            mCombinedCTs[ctIdx]->Del(record->GetItemSetAt(ctIdx), false, false);
            mCombinedCTs[ctIdx]->CoverDB(mCombinedCTs[ctIdx]->GetCurStats());
            mChangedSplitSizes[ctIdx] = mCombinedCTs[ctIdx]->CalcEncodedDbSize();

            itr = mSingleCTs[ctIdx]->Del(itr, false);
            mChangedCTSizes[ctIdx] = CalcEncodedSizeOfCTFromSingleCT(ctIdx);

            if (CalcGainFromIndividualSizes() >= 0) {
                pruneAgain = true;
                RemoveCandidateFromPool(record->dnid);
                mCombinedCTs[ctIdx]->CommitLastDel(true);
                mCurrentSplitSizes[ctIdx] = mChangedSplitSizes[ctIdx];

                mSingleCTs[ctIdx]->CommitLastDel();
                mCurrentCTSizes[ctIdx] = mChangedCTSizes[ctIdx];
            } else {
                mCombinedCTs[ctIdx]->RollbackLastDel();
                mChangedSplitSizes[ctIdx] = mCurrentSplitSizes[ctIdx];
                mSingleCTs[ctIdx]->RollbackLastDel();
                mChangedCTSizes[ctIdx] = mCurrentCTSizes[ctIdx];
                ++itr;
            }
        } else {
            ++itr;
        }
    }
    return pruneAgain;
}

void DiffNorm::ReEstimateGainsOfCandidates() {
    if (mReEstimateAllCandidates)   ReEstimateGainsOfAllCandidates();
    else                            ReEstimateGainsOfChangedCandidates();
}

void DiffNorm::ReEstimateGainsOfAllCandidates() {
    vector<DNCandidate*>::iterator itr;
    for (itr = mCandidates.begin(); itr != mCandidates.end(); ++itr) {
        DNCandidate *candidate = *itr;
        EstimateGain(candidate);
    }
}

void DiffNorm::ReEstimateGainsOfChangedCandidates() {
    // Our gain estimation relies on the usage counts inside the materialized 
    // code table. That's why we cannot trust the usage count of the records in 
    // the pseudo code table. 
    vector<DNCandidate*>::iterator itr;
    for (itr = mCandidates.begin(); itr != mCandidates.end(); ++itr) {
        DNCandidate *candidate = *itr;
        if (candidate->mergedIs->Intersects(mCandidate->mergedIs))
            EstimateGain(candidate);
    }
}

void DiffNorm::RejectCandidate() {
    mRejected.insert(mCandidate->GetDNID());
}

bool DiffNorm::RejectedBefore(const string& dnid) {
    return mRejected.find(dnid) != mRejected.end();
}

bool DiffNorm::RejectedBefore(ItemSet *is) {
    return mRejected.find(is->GetDNID()) != mRejected.end();
}

void DiffNorm::RemoveCandidateFromPool(const string &recordId) {
    vector<DNCandidate*>::iterator itr;
    for (itr = mCandidates.begin(); itr != mCandidates.end();) {
        DNCandidate* candidate = *itr;
        string candidateId = candidate->GetDNID();
        if (candidate->GetLeftDNID() == recordId || candidate->GetRightDNID() == recordId) {
            mRejected.insert(candidateId);
            mGenerated.erase(candidateId);
            delete *itr;
            itr = mCandidates.erase(itr);
        } else {
            ++itr;
        }
    }
}

void DiffNorm::RollbackRejectedChanges() {
    if (mCandidate->leftCTIdx == mCandidate->rightCTIdx && mCandidate->leftCTIdx == mNonSingletonCCTIdx) {
        // CT_i + CT_i
        if (mAddCandidateToCCT) {
            for (uint32 i = 0; i < mNumDBSplits; ++i)
                mSingleCTs[i]->RollbackLastAdd();
        } else {
            for (uint32 i=0; i<mNumDBSplits+1; ++i) {
                if (!Exists(mCommitCTIndices, i)) {
                    mSingleCTs[i]->RollbackLastAdd();
                    if (i != mNonSingletonCCTIdx)   mCombinedCTs[i]->RollbackAdd();
                }
            }
        }
    } else if (!mCandidateAccepted) {
        uint32 ctAdded;
        if (mCandidate->leftCTIdx == mCandidate->rightCTIdx && mCandidate->leftCTIdx != mNonSingletonCCTIdx)
            ctAdded = mCandidate->leftCTIdx;    // CT_i + CT_i
        else
            ctAdded = mCandidate->leftCTIdx != mNonSingletonCCTIdx ? mCandidate->leftCTIdx : mCandidate->rightCTIdx;    // CT_i + CT_Omega
        
        mSingleCTs[ctAdded]->RollbackLastAdd();
        mCombinedCTs[ctAdded]->RollbackAdd();
    }
    
    ShallowCopy(mChangedCTSizes, mCurrentCTSizes);
    ShallowCopy(mChangedSplitSizes, mCurrentSplitSizes);
}

void DiffNorm::Run() {
    InitializeSingleCTs();
    RunStrategy();
    WriteReports();
}

void DiffNorm::RunStrategy() {
    if (mRunStrategy == RCPA)   RunRegenerateCandidatesPostAcceptanceStrategy();
    else                        RunRegenerateCandidatesAfterTestingAllStrategy();
}

void DiffNorm::RunRegenerateCandidatesAfterTestingAllStrategy() {
    NoteDownCompressionStartTime();
    do {
        GenerateCandidates();
        while (!mCandidates.empty()) {
            LoadNextCandidate();
            AddCandidateToAllCTs();
            CheckGainInEachCTs();
            CommitAcceptedChanges();
            RollbackRejectedChanges();
            if (mCandidateAccepted) {
                AcceptCandidate();
                WriteProgressToScreen();
                if (mPruneStrategy == PostAcceptPruneStrategy) PostPrune();
            } else {
                RejectCandidate();
            }
            UpdateBookKeepingStuff();
        }
    } while (mGenerateMoreCandidates);
    NoteDownCompressionEndTime();
}

void DiffNorm::RunRegenerateCandidatesPostAcceptanceStrategy() {
    NoteDownCompressionStartTime();
    GenerateCandidates();
    while (!mCandidates.empty()) {
        LoadNextCandidate();
        AddCandidateToAllCTs();
        CheckGainInEachCTs();
        CommitAcceptedChanges();
        RollbackRejectedChanges();
        if (mCandidateAccepted) {
            AcceptCandidate();
            WriteProgressToScreen();
            if (mPruneStrategy == PostAcceptPruneStrategy) PostPrune();
            UpdateCandidateStats();
            ReEstimateGainsOfCandidates();
            UpdateBookKeepingStuff();
            GenerateCandidates();
        } else {
            RejectCandidate();
            UpdateCandidateStats();
            UpdateBookKeepingStuff();
        }
    } 
    NoteDownCompressionEndTime();
}

void DiffNorm::UpdateBookKeepingStuff() {
    mGenerated.erase(mCandidate->GetDNID());

    ShallowCopy(mChangedCTSizes, mCurrentCTSizes);
    ShallowCopy(mChangedSplitSizes, mCurrentSplitSizes);

    delete mCandidate;
    mCandidate = NULL;
}

void DiffNorm::UpdateCandidateStats() {
    mCandidateStats.push_back(CandidateStats(mCandidate->estGain, mCandidate->actGain, mCurTotalSize, mCandidate->estUsgCount, mCandidateAccepted, mCandidate->GetDNID()));
}

void DiffNorm::UpdateSingletonsCCT() {
    mDB->EnableFastDBOccurences();
    uint32 numAlphabets = mDB->GetAlphabetSize();
    PseudoCT *ct = new PseudoCT();
    for (uint32 i = 0; i < numAlphabets; ++i) {
        uint32 lengthOfTidList = mDB->GetAlphabetNumRows()[i];
        if (lengthOfTidList < mMinSup)
            continue;
        
        uint32 *tidList = mDB->GetAlphabetDbOccs()[i];
        uint32* rows = new uint32[lengthOfTidList];
        memcpy(rows, tidList, lengthOfTidList * sizeof (uint32));
        ItemSet *singleton = ItemSet::CreateSingleton(mDB->GetDataType(), i, numAlphabets);
        singleton->SetDBOccurrence(rows, lengthOfTidList);
        singleton->SetSupport(lengthOfTidList);
        singleton->SetUsageCount(lengthOfTidList);
        singleton->SetUsage(tidList, lengthOfTidList);
        singleton->SetStandardLength(mDB->GetStdLength(i));
#if _MSC_VER <= 1600
		singleton->SetDNID(to_string((unsigned long long) i));
#else
		singleton->SetDNID(to_string(i));
#endif
        singleton->SetUniqueID(i);
        IsRecord *singletonRecord = new IsRecord(singleton, mNumDBSplits);
		delete singleton;
        ct->Add(singletonRecord);
    }
    mSingleCTs.push_back(ct);
}

void DiffNorm::WriteCandidateStatsToDisk() {
    string filename = mOutDir + "candidate-stats.csv";
    FILE *file;
    if (fopen_s(&file, filename.c_str(), "w") != 0) {
        printf("Cannot open report file for writing!\nFilename: %s\n", filename.c_str());
        if (fopen_s(&file, filename.c_str(), "w") != 0) {
            printf("Second try failed as well..\n");
            return;
        }
    }
    
    fprintf(file, "candidate, est. gain, act. gain, accepted, size, est. usg\n");
    vector<CandidateStats>::iterator itr;
    for (itr=mCandidateStats.begin(); itr!=mCandidateStats.end(); ++itr) {
        CandidateStats stats = *itr;
        fprintf(file, "%s,%f,%f,%d,%f,%d\n", stats.id.c_str(), stats.estGain, stats.actGain, stats.accepted, stats.size, stats.estUsgCount);
    }
    fclose(file);
}

void DiffNorm::WriteCTsToDisk() {    
    uint32 supp, usgCount;
    double cost;
    char buf[255];
    
    CalcImportanceOfItemSetsInAllCTs();
    
    sprintf_s(buf, 255, "%sct-%s-%d-%" I64d ".ct", mOutDir.c_str(), mTag.c_str(), mMinSup, GetNumAcceptedCandidates() + GetNumRejectedCandidates());
    string filename = buf;
    FILE *file;
    if (fopen_s(&file, filename.c_str(), "w") != 0) {
        printf("Cannot open CodeTable file for writing!\nFilename: %s\n", filename.c_str());
        if (fopen_s(&file, filename.c_str(), "w") != 0) {
            printf("Second try failed as well..\n");
            return;
        }
    }

    fprintf(file, "diffnorm_ct-1.0\n");
    fprintf(file, "min_supp=%d\tnum_alphabets=%" I64d "\n", mMinSup, mDB->GetAlphabetSize());
    for (uint32 k = 0; k <= mNonSingletonCCTIdx; ++k) {
        sort(mSingleCTs.at(k)->records.begin(), mSingleCTs.at(k)->records.end(), CTRecordOrder(k, mNonSingletonCCTIdx));
        fprintf(file, "===========================================================\n");
        
        if (k == mNonSingletonCCTIdx)
            fprintf(file, "CT_Common :: #itemsets=%d                                   \n", mSingleCTs.at(k)->NumRecords());
        else
            fprintf(file, "CT_%d :: #itemsets=%d                                   \n", k, mSingleCTs.at(k)->NumRecords());
        fprintf(file, "===========================================================\n");

        string rows = "";
        for (recordsItr itr = mSingleCTs.at(k)->records.begin(); itr != mSingleCTs.at(k)->records.end(); ++itr) {
            IsRecord *record = (*itr);
            vector<uint32> ids;
            string isId = record->dnid;
            stringstream ss(isId);
            while (getline(ss, isId, '-'))
                ids.push_back(stoi(isId));

            vector<uint32>::iterator intItr;
            for (intItr = ids.begin(); intItr != ids.end(); ++intItr) {
                uint32 ficId = *intItr;
                
                if (mReportTranslated)
                    ficId = mDB->GetItemTranslator()->BitToValue(ficId);
                
                #if _MSC_VER <= 1600
                    rows += to_string((unsigned long long) ficId) + " ";
                #else
                    rows += to_string(ficId) + " ";
                #endif
            }

            if (k == mNonSingletonCCTIdx) {
                mDB->CalcSupport(record->GetItemSetAt(0));
                supp = record->GetItemSetAt(0)->GetSupport();
                usgCount = GetUsageCount(record, k);
                cost = record->costOfDeletion;
            } else {
                ItemSet *is = record->GetItemSetAt(k);
                supp = is->GetSupport();
                usgCount = is->GetUsageCount();
                cost = record->costOfDeletion;
            }
                
            #if _MSC_VER <= 1600
                rows += "\t(" + to_string((unsigned long long)usgCount) + " , " + to_string((unsigned long long)supp) + " , " + to_string((unsigned long long) cost) + ")\n";
            #else
                rows += "\t(" + to_string(usgCount) + " , " + to_string(supp) + " , " + to_string(cost) + ")\n";
            #endif
        }
        fprintf(file, "%s", rows.c_str());
    }
    fclose(file);
    printf(" * Output Directory: \t%s", mOutDir.c_str());
}

void DiffNorm::WriteProgressToScreen() {
    if(Bass::GetOutputLevel() > 0) {
        uint32 numprinted = printf_s(" * Progress:\t\t%d, %.00fb (%.01fb) (%fs)", mAccepted.size(), mCurTotalSize, mCandidate->actGain, omp_get_wtime() - mCompressionStartTime);
        if(Bass::GetOutputLevel() == 3)
            printf("\n");
        else {
            if(numprinted < 80)
                for(uint32 i=0; i< (80-numprinted); i++)
                    printf(" ");
            printf("\r");
        }
        fflush(stdout);
    }
}

void DiffNorm::WriteReportToDisk() {
    double currentTotalSize = CalcTotalEncodedSize();
    double compressionRatio = currentTotalSize * 100.0 / mTotalSizeInStart;
    string filename = mOutDir + "report.txt";
    FILE *file;
    if (fopen_s(&file, filename.c_str(), "w") != 0) {
        printf("Cannot open report file for writing!\nFilename: %s\n", filename.c_str());
        if (fopen_s(&file, filename.c_str(), "w") != 0) {
            printf("Second try failed as well..\n");
            return;
        }
    }
    
    uint32 numRecord, numIs=0;
    string numIsPerCT = "[";
    for (uint32 i=0; i<=mNonSingletonCCTIdx; ++i) {
        numRecord = mSingleCTs[i]->NumRecords();
        
        #if _MSC_VER <= 1600
            numIsPerCT += to_string((unsigned long long)numRecord);
        #else
            numIsPerCT += to_string(numRecord);
        #endif

        if (i != mNonSingletonCCTIdx)   numIsPerCT += ", ";
        else                            numIsPerCT += "]";
            
        numIs += numRecord;
    }
    
    fprintf(file, "============diffnorm_ct-1.0============\n");
    fprintf(file, "Minimum support                 = %d\n", mMinSup);
    fprintf(file, "Number of splits                = %d\n", mNumDBSplits);
    fprintf(file, "Number of codetables            = %d\n", mNumDBSplits + 1);
    fprintf(file, "Total encoded size (@start)     = %f\n", mTotalSizeInStart);
    fprintf(file, "Total encoded size (@end)       = %f\n", currentTotalSize);
    fprintf(file, "Compression ratio achieved      = %f %c\n", compressionRatio, 37);
    fprintf(file, "Number of candidates generated  = %" I64d "\n", mNumGeneratedCandidates);
    fprintf(file, "Number of candidates rejected   = %" I64d "\n", GetNumRejectedCandidates());
    fprintf(file, "Number of candidates accepted   = %" I64d "\n", GetNumAcceptedCandidates());
    fprintf(file, "Number of itemsets in CTs       = %" I64d "\n", numIs);
    fprintf(file, "Number of itemsets Per CTs      = %s\n", numIsPerCT.c_str());
    fprintf(file, "Time to compress                = %f seconds\n", CalcCompressionTime());
    fprintf(file, "-----------------------------------------------------------\n");
    fprintf(file, "%15s | %15s | %15s\n", "Code Table", "Encoded Size", "#Itemsets");
    fprintf(file, "-----------------------------------------------------------\n");
    for (uint32 i = 0; i < mNumDBSplits + 1; ++i) {
        if (i == mNonSingletonCCTIdx)
            fprintf(file, "%15s | %15f | %15d\n", "Common", mCurrentCTSizes[i], mSingleCTs[i]->NumRecords());
        else
            fprintf(file, "%15d | %15f | %15d\n", i, mCurrentCTSizes[i], mSingleCTs[i]->NumRecords());
    }
    fprintf(file, "-----------------------------------------------------------\n\n");
    fprintf(file, "-----------------------------------------------------------\n");
    fprintf(file, "%15s | %15s | %15s\n", "Database Split", "Encoded Size", "#Transactions");
    fprintf(file, "-----------------------------------------------------------\n");
    for (uint32 i = 0; i < mNumDBSplits; ++i)
        fprintf(file, "%15d | %15f | %15d\n", i, mCurrentSplitSizes[i], mCombinedCTs[i]->GetDatabase()->GetNumTransactions());
    fprintf(file, "-----------------------------------------------------------\n");
    fclose(file);
}

void DiffNorm::WriteReports() {
    WriteCTsToDisk();
    if (mWriteStatsFile)    WriteCandidateStatsToDisk();
    if (mWriteReportFile)   WriteReportToDisk();
}

#endif // BLOCK_DIFFNORM
